import requests
from datetime import datetime
from lxml import etree
from tinydb import where
from common import UID,HandleTmpList,parseContentToName,ProductToGroup,sess
SPIDERNAME='焦联网'
def parseJiaoLianPubTime(time_str):
    #(2020-12-25 09:38:40)
    return time_str.strip().strip('(').strip(')')

def parseJiaoLianContent(articleUrl):
    r=sess.get(articleUrl)
    selector=etree.HTML(r.text)
    ele_ls=selector.xpath("//div[@class='rich_media_content']")
    if ele_ls:
        content=ele_ls[0].xpath('string(.)')
    else:
        print(articleUrl,'+'*10,'没有取到文章内容')
        content=''
    return content.strip()

def getJiaoLianArticleList(articleCol,BeCrawledUrlList):
    r=sess.get('http://www.jiaolianwang.com/index.php?controller=site&action=article')
    selector=etree.HTML(r.text)
    eleList=selector.xpath("//div[@class='y-newslist-right-con']/ul/li")
    baseUrl='http://www.jiaolianwang.com'
    temp_article_ls=[]
    for ele in  eleList:

        articleUrl=baseUrl+ele.xpath('./a/@href')[0]
        #判断是否已经爬取过，如果是，跳出循环
        if len(articleCol.search(where('url')==articleUrl)):break
        title=ele.xpath('./a/text()')[0]
        publicTime=parseJiaoLianPubTime(ele.xpath('./a/span/text()')[0])
        temp_dict={'tags':[SPIDERNAME],'score':0,'uid':UID()}
        temp_dict['title']=title.strip()
        temp_dict['articleFrom']=SPIDERNAME
        temp_dict['url']=articleUrl.strip()
        temp_dict['publicTime']=publicTime.strip()
         #文章内容
        content=parseJiaoLianContent(articleUrl)
        temp_dict['content']=content
        #定文章所属期货品种,板块
        # n=parseContentToName(title+content)
        n='焦炭'
        if n:
            print(SPIDERNAME,'   ',title,"  ",n)
            temp_dict['product_name']=n
            temp_dict['group']=ProductToGroup[n]
        else:
            print("………………………………未找到品种名称，可能异常")
            temp_dict['product_name']=''
            temp_dict['group']=''
        
        temp_article_ls.append(temp_dict)

    #注意缩进不要错
    HandleTmpList(temp_article_ls,articleCol,SPIDERNAME)

